﻿/* 
 * Copyright (c) Intel Corporation
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * -- Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * -- Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * -- Neither the name of the Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE INTEL OR ITS
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Web.UI.HtmlControls;
using System.Xml;
using System.Xml.XPath;
using ExtensionLoader;
using ExtensionLoader.Config;
using OpenMetaverse.Http;
using CableBeachMessages;
using OpenMetaverse.StructuredData;

namespace WorldServer.Extensions
{
    public class TrustAllCertificatePolicy : ICertificatePolicy
    {
        public TrustAllCertificatePolicy() { }

        public bool CheckValidationResult(ServicePoint sp, X509Certificate cert, WebRequest req, int problem)
        {
            return true;
        }

        public static bool TrustAllCertificateHandler(Object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
        {
            return true;
        }
    }

    public class SimpleServices : IExtension<WorldServer>, IServiceProvider
    {
        const int REQUEST_TIMEOUT = 1000 * 30;

        WorldServer server;
        ServiceCollection worldServices = new ServiceCollection();

        public SimpleServices()
        {
            ServicePointManager.CertificatePolicy = new TrustAllCertificatePolicy();
            //ServicePointManager.ServerCertificateValidationCallback = TrustAllCertificatePolicy.TrustAllCertificateHandler;
        }

        public bool Start(WorldServer server)
        {
            this.server = server;

            #region Config Loading

            string serviceFile = null;

            try
            {
                IConfig serviceConfig = server.ConfigFile.Configs["Services"];
                serviceFile = serviceConfig.GetString("ServiceDefinitions");

                OSDMap servicesMap = OSDParser.DeserializeJson(
                    new FileStream(WorldServer.DATA_DIR + serviceFile, FileMode.Open, FileAccess.Read)) as OSDMap;

                foreach (KeyValuePair<string, OSD> serviceEntry in servicesMap)
                {
                    OSDMap serviceMap = serviceEntry.Value as OSDMap;
                    OSDArray capsArray = serviceMap["capabilities"] as OSDArray;

                    Uri serviceIdentifier = new Uri(serviceEntry.Key);

                    // Convert the array of names such as "get_asset" to full identifiers such as "http://openmetaverse.org/services/assets/get_asset"
                    // and put them into a dictionary as keys. The values would be the service endpoints for each capability,
                    // which are not filled in yet
                    Dictionary<Uri, Uri> capabilities = new Dictionary<Uri, Uri>(capsArray.Count);
                    foreach (OSD cap in capsArray)
                        capabilities.Add(new Uri(serviceIdentifier.ToString().TrimEnd('/') + '/' + cap.AsString()), null);

                    Uri type = new Uri(serviceEntry.Key);
                    Uri location = serviceMap["location"].AsUri();
                    bool trusted = serviceMap["is_trusted"].AsBoolean();
                    bool canOverride = !serviceMap["no_override"].AsBoolean();

                    // Use the Link-Based Resource Descriptor Discovery (LRDD) process to get OAuth/SeedCap endpoints for this service
                    Service service = CreateServiceFromLRDD(location, type, trusted, canOverride);

                    if (service != null)
                    {
                        service.Capabilities = capabilities;

                        Logger.Info("[SimpleServices] Adding service requirement: " + service.ToString());
                        worldServices.Add(service.Identifier, service);
                    }
                    else
                    {
                        Logger.Error("[SimpleServices] Failed to discover service type " + type + " at endpoint " + location);
                        return false;
                    }
                }
            }
            catch (Exception ex)
            {
                if (serviceFile != null)
                    Logger.Error("[SimpleServices] Failed to load service definitions from " + serviceFile + ": " + ex);
                else
                    Logger.Error("[SimpleServices] Failed to load [Services] section from " + WorldServer.CONFIG_FILE + ": " + ex);
                return false;
            }
            #endregion Config Loading

            return true;
        }

        public void Stop()
        {
        }

        public void GetServices(Uri identity, bool fetchTrustedCapabilities, ref ServiceCollection requestedServices)
        {
            ServiceCollection newServices = new ServiceCollection();

            foreach (KeyValuePair<Uri, Service> entry in worldServices)
            {
                Service service;

                // Handle overriding world services with external services
                Service requestedService;
                if (entry.Value.CanOverride && requestedServices.TryGetValue(entry.Key, out requestedService))
                {
                    service = new Service(requestedService);
                    // Even if we are overriding, the world still determines which capabilities are required
                    service.Capabilities = new Dictionary<Uri, Uri>(entry.Value.Capabilities);
                }
                else
                {
                    service = new Service(entry.Value);
                }

                // Fetch capabilities from trusted services
                if (fetchTrustedCapabilities && service.IsTrusted)
                    FetchCapabilities(identity, ref service);
                
                // Add this service to the newServices collection
                newServices.Add(entry.Key, service);
            }

            // Store webdav url to SimpleServiceGetter cache if its initialized
            if (SimpleServicesGetter.isInitialized)
            {
                Service service;
                Uri webdavUrl, webdavUrlSet;
                if ( newServices.TryGetValue(new Uri(CableBeachServices.FILESYSTEM_WEBDAV), out service) )
                {
                    if (service.Capabilities.TryGetValue(new Uri(CableBeachServices.FILESYSTEM_GET_WEBDAV_ROOT), out webdavUrl))
                    {
                        if ( SimpleServicesGetter.WebDavServiceCache.TryGetValue(identity, out webdavUrlSet) )
                        {
                            Logger.Debug("[SimpleServices] Identitys webdav service endpoint already in local cache, skipping storage");
                        }
                        else
                        {
                            SimpleServicesGetter.WebDavServiceCache.Add(identity, webdavUrl);
                            Logger.Debug("[SimpleServices] Added webdav service endpoint to SimpleServicesGetter cache for identity " + identity.ToString());
                        }
                    }
                }
            }
             
            requestedServices = newServices;
        }

        public static Service CreateServiceFromLRDD(Uri serviceLocation, Uri serviceType, bool isTrusted, bool allowOverride)
        {
            if (serviceLocation == null)
            {
                Logger.Error("[SimpleServices] Cannot fetch services from an empty location. Check your WorldServer.Services.txt file");
                return null;
            }

            HttpWebResponse response;
            Uri xrdUrl = null;
            MemoryStream xrdStream = null;

            MemoryStream serviceStream = FetchWebDocument(serviceLocation, "text/html,application/xhtml+xml,application/xrd+xml,application/xml,text/xml", out response);

            if (serviceStream != null)
            {
                if (IsXrdDocument(response.ContentType.ToLowerInvariant(), serviceStream))
                {
                    // We fetched an XRD document directly, skip ahead
                    xrdUrl = serviceLocation;
                    xrdStream = serviceStream;
                }
                else
                {
                    #region LRDD

                    // 1. Check the HTTP headers for Link: <...>; rel="describedby"; ...
                    xrdUrl = FindXrdDocumentLocationInHeaders(response.Headers);

                    // 2. Check the document body for <link rel="describedby" ...>
                    if (xrdUrl == null)
                        xrdUrl = FindXrdDocumentLocationInHtmlMetaTags(GetStreamString(serviceStream));

                    // 3. TODO: Try and grab the /host-meta document
                    if (xrdUrl == null)
                        xrdUrl = FindXrdDocumentLocationFromHostMeta(new Uri(serviceLocation, "/host-meta"));

                    // 4. Fetch the XRD document
                    if (xrdUrl != null)
                    {
                        serviceStream = FetchWebDocument(xrdUrl, "application/xrd+xml,application/xml,text/xml", out response);

                        if (serviceStream != null && IsXrdDocument(response.ContentType.ToLowerInvariant(), serviceStream))
                            xrdStream = serviceStream;
                        else
                            Logger.Error("[SimpleServices] XRD fetch from " + xrdUrl + " failed");

                        response.Close();
                    }

                    #endregion LRDD
                }

                response.Close();

                if (xrdStream != null)
                    return XrdDocumentToService(xrdStream, xrdUrl, serviceType, isTrusted, allowOverride);
            }
            else
            {
                Logger.Error("[SimpleServices] Discovery on endpoint " + serviceLocation + " failed");
            }

            return null;
        }

        public static MemoryStream FetchWebDocument(Uri location, string acceptTypes, out HttpWebResponse response)
        {
            const int MAXIMUM_BYTES = 1024 * 1024;
            const int TIMEOUT = 10000;
            const int READ_WRITE_TIMEOUT = 1500;
            const int MAXIMUM_REDIRECTS = 10;

            try
            {
                HttpWebRequest request = UntrustedHttpWebRequest.Create(location, true, READ_WRITE_TIMEOUT, TIMEOUT, MAXIMUM_REDIRECTS);
                request.Accept = acceptTypes;

                response = (HttpWebResponse)request.GetResponse();
                MemoryStream documentStream;

                using (Stream networkStream = response.GetResponseStream())
                {
                    documentStream = new MemoryStream(response.ContentLength < 0 ? 4 * 1024 : Math.Min((int)response.ContentLength, MAXIMUM_BYTES));
                    networkStream.CopyTo(documentStream, MAXIMUM_BYTES);
                    documentStream.Seek(0, SeekOrigin.Begin);
                }

                if (response.StatusCode == HttpStatusCode.OK)
                    return documentStream;
                else
                    Logger.ErrorFormat("[SimpleServices] HTTP error code {0} returned while fetching {1}", response.StatusCode, location);
            }
            catch (Exception ex)
            {
                Logger.ErrorFormat("[SimpleServices] HTTP error while fetching {0}: {1}", location, ex.Message);
            }

            response = null;
            return null;
        }

        public static bool IsXrdDocument(string contentType, Stream documentStream)
        {
            if (String.IsNullOrEmpty(contentType))
                return false;

            if (contentType == "application/xrd+xml" || contentType == "application/xrds+xml")
                return true;

            if (contentType.EndsWith("xml"))
            {
                documentStream.Seek(0, SeekOrigin.Begin);
                XmlReader reader = XmlReader.Create(documentStream);
                while (reader.Read() && reader.NodeType != XmlNodeType.Element)
                {
                    // Skip over non-element nodes
                }

                if (reader.Name == "XRD")
                    return true;
            }

            return false;
        }

        public static string GetStreamString(Stream stream)
        {
            if (stream != null)
            {
                StreamReader reader = new StreamReader(stream);
                string value = reader.ReadToEnd();
                stream.Seek(0, SeekOrigin.Begin);
                return value;
            }

            return null;
        }

       public static Uri FindXrdDocumentLocationInHtmlMetaTags(string html)
        {
            foreach (HtmlLink linkTag in HtmlParser.HeadTags<HtmlLink>(html))
            {
                string rel = linkTag.Attributes["rel"];
                if (rel != null && rel.Equals("describedby", StringComparison.OrdinalIgnoreCase))
                {
                    Uri uri;
                    if (Uri.TryCreate(linkTag.Href, UriKind.Absolute, out uri))
                        return uri;
                }
            }

            return null;
        }

        public static Uri FindXrdDocumentLocationInHeaders(WebHeaderCollection headers)
        {
            Uri xrdUrl = null;

            string[] links = headers.GetValues("link");
            if (links != null && links.Length > 0)
            {
                for (int i = 0; i < links.Length; i++)
                {
                    string link = links[i];
                    if (link.Contains("rel=\"describedby\""))
                    {
                        if (Uri.TryCreate(Regex.Replace(link, @"^.*<(.*?)>.*$", "$1"), UriKind.Absolute, out xrdUrl))
                            break;
                    }
                }
            }

            return xrdUrl;
        }

        static Uri FindXrdDocumentLocationFromHostMeta(Uri hostMetaLocation)
        {
            // TODO: Implement this
            return null;
        }

        static Service XrdDocumentToService(Stream xriStream, Uri xrdLocation, Uri serviceType, bool isTrusted, bool allowOverride)
        {
            XrdParser parser = new XrdParser(xriStream);
            XrdDocument xrd = parser.Document;

            Uri SEED_CAPABILITY = new Uri(CableBeachServices.SEED_CAPABILITY);
            Uri OAUTH_INITIATE = new Uri(CableBeachServices.OAUTH_INITIATE);
            Uri OAUTH_AUTHORIZE = new Uri(CableBeachServices.OAUTH_AUTHORIZE);
            Uri OAUTH_TOKEN = new Uri(CableBeachServices.OAUTH_TOKEN);

            Uri seedCap = null, oauthRequest = null, oauthAuthorize = null, oauthAccess = null;

            // Grab the endpoints we are looking for from the XRD links
            for (int i = 0; i < xrd.Links.Count; i++)
            {
                XrdLink link = xrd.Links[i];

                if (link.Relation == SEED_CAPABILITY)
                    seedCap = GetHighestPriorityUri(link.Uris);
                else if (link.Relation == OAUTH_INITIATE)
                    oauthRequest = GetHighestPriorityUri(link.Uris);
                else if (link.Relation == OAUTH_AUTHORIZE)
                    oauthAuthorize = GetHighestPriorityUri(link.Uris);
                else if (link.Relation == OAUTH_TOKEN)
                    oauthAccess = GetHighestPriorityUri(link.Uris);
            }

            // Check that this service actually fulfills the type of service we need
            bool serviceMatch = false;
            for (int i = 0; i < xrd.Types.Count; i++)
            {
                if (serviceType.ToString() == xrd.Types[i])
                {
                    serviceMatch = true;
                    break;
                }
            }
            if (!serviceMatch)
            {
                Logger.Error("[SimpleServices] Discovery failed at endpoint " + xrdLocation + ", does not provide the service type " + serviceType);
                return null;
            }

            // Check that either a seed cap or all of the OAuth endpoints were fetched
            bool hasSeedCap = (seedCap != null);
            bool hasOAuth = (oauthRequest != null && oauthAuthorize != null && oauthAccess != null);
            if (!hasSeedCap && !hasOAuth)
            {
                Logger.Error("[SimpleServices] Discovery failed at endpoint " + xrdLocation + ", incomplete list of required service endpoints");
                return null;
            }

            // Success, return the service
            return new Service(
                serviceType,
                xrdLocation,
                seedCap,
                oauthRequest,
                oauthAuthorize,
                oauthAccess,
                isTrusted,
                allowOverride);
        }

        static Uri GetHighestPriorityUri(List<XrdUri> uris)
        {
            Uri topUri = null;
            int topPriority = Int32.MaxValue;

            for (int i = 0; i < uris.Count; i++)
            {
                XrdUri uri = uris[i];

                // If this is the highest priority URI, set the top URI to this URI
                if (uri.Priority >= 0 && uri.Priority < topPriority)
                {
                    topUri = uri.Uri;
                    topPriority = uri.Priority;
                }

                // If this URI has no priority and no top URI has been set yet, set the top URI to this URI
                if (topUri == null && uri.Priority < 0)
                    topUri = uri.Uri;
            }

            return topUri;
        }

        static void FetchCapabilities(Uri identity, ref Service service)
        {
            RequestCapabilitiesMessage message = new RequestCapabilitiesMessage();
            message.Identity = identity;
            message.Capabilities = service.GetUnassociatedCapabilities();

            Logger.Debug("[SimpleServices] Requesting " + message.Capabilities.Length + " capabilities from seed capability at " + service.SeedCapability);

            CapsClient request = new CapsClient(service.SeedCapability);
            OSDMap responseMap = request.GetResponse(message.Serialize(), OSDFormat.Json, REQUEST_TIMEOUT) as OSDMap;

            if (responseMap != null)
            {
                RequestCapabilitiesReplyMessage reply = new RequestCapabilitiesReplyMessage();
                reply.Deserialize(responseMap);

                Logger.Info("[SimpleServices] Fetched " + reply.Capabilities.Count + " capabilities from seed capability at " + service.SeedCapability);

                foreach (KeyValuePair<Uri, Uri> entry in reply.Capabilities)
                    service.Capabilities[entry.Key] = entry.Value;
            }
            else
            {
                Logger.Warn("[SimpleServices] Failed to fetch capabilities from seed capability at " + service.SeedCapability);
            }
        }
    }

    internal static class StreamUtilities
    {
        /// <summary>
        /// Copies the contents of one stream to another.
        /// </summary>
        /// <param name="copyFrom">The stream to copy from, at the position where copying should begin.</param>
        /// <param name="copyTo">The stream to copy to, at the position where bytes should be written.</param>
        /// <param name="maximumBytesToCopy">The maximum bytes to copy.</param>
        /// <returns>The total number of bytes copied.</returns>
        /// <remarks>
        /// Copying begins at the streams' current positions.
        /// The positions are NOT reset after copying is complete.
        /// </remarks>
        internal static int CopyTo(this Stream copyFrom, Stream copyTo, int maximumBytesToCopy)
        {
            byte[] buffer = new byte[1024];
            int readBytes;
            int totalCopiedBytes = 0;

            while ((readBytes = copyFrom.Read(buffer, 0, Math.Min(1024, maximumBytesToCopy))) > 0)
            {
                int writeBytes = Math.Min(maximumBytesToCopy, readBytes);
                copyTo.Write(buffer, 0, writeBytes);
                totalCopiedBytes += writeBytes;
                maximumBytesToCopy -= writeBytes;
            }

            return totalCopiedBytes;
        }
    }

    internal static class UntrustedHttpWebRequest
    {
        private static readonly ICollection<string> allowableSchemes = new List<string> { "http", "https" };

        public static HttpWebRequest Create(Uri uri, bool allowLoopback, int readWriteTimeoutMS, int timeoutMS, int maximumRedirects)
        {
            if (uri == null)
                throw new ArgumentNullException("uri");

            if (!IsUriAllowable(uri, allowLoopback))
                throw new ArgumentException("Uri " + uri + " was rejected");

            HttpWebRequest httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(uri);
            httpWebRequest.MaximumAutomaticRedirections = maximumRedirects;
            httpWebRequest.ReadWriteTimeout = readWriteTimeoutMS;
            httpWebRequest.Timeout = timeoutMS;
            httpWebRequest.KeepAlive = false;

            return httpWebRequest;
        }

        /// <summary>
        /// Determines whether a URI is allowed based on scheme and host name.
        /// No requireSSL check is done here
        /// </summary>
        /// <param name="allowLoopback">True to allow loopback addresses to be used</param>
        /// <param name="uri">The URI to test for whether it should be allowed.</param>
        /// <returns>
        /// 	<c>true</c> if [is URI allowable] [the specified URI]; otherwise, <c>false</c>.
        /// </returns>
        private static bool IsUriAllowable(Uri uri, bool allowLoopback)
        {
            if (!allowableSchemes.Contains(uri.Scheme))
            {
                Logger.WarnFormat("[SimpleServices] Rejecting URL {0} because it uses a disallowed scheme.", uri);
                return false;
            }

            // Try to interpret the hostname as an IP address so we can test for internal
            // IP address ranges.  Note that IP addresses can appear in many forms 
            // (e.g. http://127.0.0.1, http://2130706433, http://0x0100007f, http://::1
            // So we convert them to a canonical IPAddress instance, and test for all
            // non-routable IP ranges: 10.*.*.*, 127.*.*.*, ::1
            // Note that Uri.IsLoopback is very unreliable, not catching many of these variants.
            IPAddress hostIPAddress;
            if (IPAddress.TryParse(uri.DnsSafeHost, out hostIPAddress))
            {
                byte[] addressBytes = hostIPAddress.GetAddressBytes();

                // The host is actually an IP address.
                switch (hostIPAddress.AddressFamily)
                {
                    case System.Net.Sockets.AddressFamily.InterNetwork:
                        if (!allowLoopback && (addressBytes[0] == 127 || addressBytes[0] == 10))
                        {
                            Logger.WarnFormat("[SimpleServices] Rejecting URL {0} because it is a loopback address.", uri);
                            return false;
                        }
                        break;
                    case System.Net.Sockets.AddressFamily.InterNetworkV6:
                        if (!allowLoopback && IsIPv6Loopback(hostIPAddress))
                        {
                            Logger.WarnFormat("[SimpleServices] Rejecting URL {0} because it is a loopback address.", uri);
                            return false;
                        }
                        break;
                    default:
                        Logger.WarnFormat("[SimpleServices] Rejecting URL {0} because it does not use an IPv4 or IPv6 address.", uri);
                        return false;
                }
            }
            else
            {
                // The host is given by name.  We require names to contain periods to
                // help make sure it's not an internal address.
                if (!allowLoopback && !uri.Host.Contains("."))
                {
                    Logger.WarnFormat("[SimpleServices] Rejecting URL {0} because it does not contain a period in the host name.", uri);
                    return false;
                }
            }

            return true;
        }

        /// <summary>
        /// Determines whether an IP address is the IPv6 equivalent of "localhost/127.0.0.1".
        /// </summary>
        /// <param name="ip">The ip address to check.</param>
        /// <returns>
        /// 	<c>true</c> if this is a loopback IP address; <c>false</c> otherwise.
        /// </returns>
        private static bool IsIPv6Loopback(IPAddress ip)
        {
            if (ip == null)
                throw new ArgumentNullException("ip");

            byte[] addressBytes = ip.GetAddressBytes();
            for (int i = 0; i < addressBytes.Length - 1; i++)
            {
                if (addressBytes[i] != 0)
                    return false;
            }

            if (addressBytes[addressBytes.Length - 1] != 1)
                return false;

            return true;
        }
    }

    /// <summary>
    /// An HTML HEAD tag parser.
    /// </summary>
    internal static class HtmlParser
    {
        /// <summary>
        /// Common flags to use on regex tests.
        /// </summary>
        private const RegexOptions Flags = RegexOptions.IgnorePatternWhitespace | RegexOptions.Singleline | RegexOptions.Compiled | RegexOptions.IgnoreCase;

        /// <summary>
        /// A regular expression designed to select tags (?)
        /// </summary>
        private const string TagExpr = "\n# Starts with the tag name at a word boundary, where the tag name is\n# not a namespace\n<{0}\\b(?!:)\n    \n# All of the stuff up to a \">\", hopefully attributes.\n(?<attrs>[^>]*?)\n    \n(?: # Match a short tag\n    />\n    \n|   # Match a full tag\n    >\n    \n    (?<contents>.*?)\n    \n    # Closed by\n    (?: # One of the specified close tags\n        </?{1}\\s*>\n    \n    # End of the string\n    |   \\Z\n    \n    )\n    \n)\n    ";

        /// <summary>
        /// A regular expression designed to select start tags (?)
        /// </summary>
        private const string StartTagExpr = "\n# Starts with the tag name at a word boundary, where the tag name is\n# not a namespace\n<{0}\\b(?!:)\n    \n# All of the stuff up to a \">\", hopefully attributes.\n(?<attrs>[^>]*?)\n    \n(?: # Match a short tag\n    />\n    \n|   # Match a full tag\n    >\n    )\n    ";

        /// <summary>
        /// A regular expression designed to select attributes within a tag.
        /// </summary>
        private static readonly Regex attrRe = new Regex("\n# Must start with a sequence of word-characters, followed by an equals sign\n(?<attrname>(\\w|-)+)=\n\n# Then either a quoted or unquoted attribute\n(?:\n\n # Match everything that's between matching quote marks\n (?<qopen>[\"\\'])(?<attrval>.*?)\\k<qopen>\n|\n\n # If the value is not quoted, match up to whitespace\n (?<attrval>(?:[^\\s<>/]|/(?!>))+)\n)\n\n|\n\n(?<endtag>[<>])\n    ", Flags);

        /// <summary>
        /// A regular expression designed to select the HEAD tag.
        /// </summary>
        private static readonly Regex headRe = TagMatcher("head", new[] { "body" });

        /// <summary>
        /// A regular expression designed to select the HTML tag.
        /// </summary>
        private static readonly Regex htmlRe = TagMatcher("html", new string[0]);

        /// <summary>
        /// A regular expression designed to remove all comments and scripts from a string.
        /// </summary>
        private static readonly Regex removedRe = new Regex(@"<!--.*?-->|<!\[CDATA\[.*?\]\]>|<script\b[^>]*>.*?</script>", Flags);

        /// <summary>
        /// Finds all the HTML HEAD tag child elements that match the tag name of a given type.
        /// </summary>
        /// <typeparam name="T">The HTML tag of interest.</typeparam>
        /// <param name="html">The HTML to scan.</param>
        /// <returns>A sequence of the matching elements.</returns>
        public static IEnumerable<T> HeadTags<T>(string html) where T : HtmlControl, new()
        {
            html = removedRe.Replace(html, string.Empty);
            Match match = htmlRe.Match(html);
            string tagName = (new T()).TagName;
            if (match.Success)
            {
                Match match2 = headRe.Match(html, match.Index, match.Length);
                if (match2.Success)
                {
                    string text = null;
                    string text2 = null;
                    Regex regex = StartTagMatcher(tagName);
                    for (Match match3 = regex.Match(html, match2.Index, match2.Length); match3.Success; match3 = match3.NextMatch())
                    {
                        int beginning = (match3.Index + tagName.Length) + 1;
                        int length = (match3.Index + match3.Length) - beginning;
                        Match match4 = attrRe.Match(html, beginning, length);
                        var headTag = new T();
                        while (match4.Success)
                        {
                            if (match4.Groups["endtag"].Success)
                            {
                                break;
                            }
                            text = match4.Groups["attrname"].Value;
                            text2 = HttpUtility.HtmlDecode(match4.Groups["attrval"].Value);
                            headTag.Attributes.Add(text, text2);
                            match4 = match4.NextMatch();
                        }
                        yield return headTag;
                    }
                }
            }
        }

        /// <summary>
        /// Generates a regular expression that will find a given HTML tag.
        /// </summary>
        /// <param name="tagName">Name of the tag.</param>
        /// <param name="closeTags">The close tags (?).</param>
        /// <returns>The created regular expression.</returns>
        private static Regex TagMatcher(string tagName, params string[] closeTags)
        {
            string text2;
            if (closeTags.Length > 0)
            {
                StringBuilder builder = new StringBuilder();
                builder.AppendFormat("(?:{0}", tagName);
                int index = 0;
                string[] textArray = closeTags;
                int length = textArray.Length;
                while (index < length)
                {
                    string text = textArray[index];
                    index++;
                    builder.AppendFormat("|{0}", text);
                }
                builder.Append(")");
                text2 = builder.ToString();
            }
            else
            {
                text2 = tagName;
            }
            return new Regex(string.Format(CultureInfo.InvariantCulture, TagExpr, tagName, text2), Flags);
        }

        /// <summary>
        /// Generates a regular expression designed to find a given tag.
        /// </summary>
        /// <param name="tagName">The tag to find.</param>
        /// <returns>The created regular expression.</returns>
        private static Regex StartTagMatcher(string tagName)
        {
            return new Regex(string.Format(CultureInfo.InvariantCulture, StartTagExpr, tagName), Flags);
        }
    }
}
